package edu.tsinghua.oxy.engineer.service.impl;

import edu.tsinghua.oxy.common.utils.HttpUtils;
import edu.tsinghua.oxy.engineer.entity.BaseInfo;
import edu.tsinghua.oxy.engineer.service.IHtmlParse;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;

/**
 * 爬取可用的代理IP和端口
 * @author sean
 * @version 1.0
 * @date 2021/7/29
 */
@Slf4j
public class HtmlParseProxy implements IHtmlParse<BaseInfo> {

    private static Set<String> resultSet = new TreeSet<>();

    private ExecutorService exe = Executors.newFixedThreadPool(50);


    /**
     * 代理列表页解析
     * @param listUrl 列表页面url
     *
     * @return
     */
    @Override
    public List<String> parseListHtml(String listUrl) {
        log.info("访问的url是：{}", listUrl);

        Document document = null;
        try {
            document = Jsoup.connect(listUrl).userAgent(new HttpUtils().getAgent()).timeout(60000).get();

            Elements trs = document.getElementsByTag("tbody").get(0).getElementsByTag("tr");
            for (Element tr : trs) {
                Elements tds = tr.getElementsByTag("td");
                String td = tds.get(0).text();
                String type = tds.get(1).text();
//                if(!type.contains("HTTPS")){
//                    continue;
//                }

                String[] split = td.split(":");
                String ip = split[0];
                int port = Integer.parseInt(split[1]);
                log.info("{}-{}-{}", ip, port, type);

                exe.submit(new CheckIp(ip, port, type));
            }
        } catch (Exception e) {
            log.warn("获取网页内容： 失败 ", e);
        }

        return null;
    }

    @Override
    public BaseInfo parseInfoHtml(String infoUrl) {
        return null;
    }

    class CheckIp implements Runnable{

        private Random r = new Random();

        private String ip;
        private int port;
        private String type;

        public CheckIp(String ip, int port, String type) {
            this.ip = ip;
            this.port = port;
            this.type = type;
        }

        @Override
        public void run() {
            try {
                Document document = Jsoup.connect("https://catalog.data.gov/dataset").proxy(ip, port)
                        .userAgent(new HttpUtils().getAgent()).timeout(10000).get();

                log.info("{}:{} 可用，类型：{}", ip, port, type);
                resultSet.add(ip + ":" + port);
            } catch (IOException e) {
            }
        }
    }

    public boolean isEnd(){
        return ((ThreadPoolExecutor) exe).getActiveCount() == 0;
    }

    public static void main(String[] args) {
        Random r = new Random();
        HtmlParseProxy htmlParseProxy = new HtmlParseProxy();
        int count = 5;
        for (int i = 1; i < count; i++) {
            htmlParseProxy.parseListHtml("http://www.xiladaili.com/http/" + i + "/");
            try {
                Thread.sleep(r.nextInt(3000));
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }

        while (!htmlParseProxy.isEnd()){
            try {
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }

        //把结果写入文件中
        log.info("查询到的所有可用代理是；{}", resultSet);
        String baseDir = System.getProperty("user.dir");
        File file = new File(baseDir + File.separator + "proxyIp.txt");
        log.info("代理ip保存位置：{}", file.getPath());
        try {
            if(!file.exists()){
                file.createNewFile();
            }

            BufferedWriter writer = new BufferedWriter(new FileWriter(file));
            for (String s : resultSet) {
                writer.write(s);
                writer.newLine();
            }

            writer.close();

            log.info("代理写入完成");
        } catch (IOException e) {
            log.error("保存代理ip出错 ", e);
        }
    }
}
